import json
from datetime import datetime
from enum import Enum
from typing import Generic, TypeVar, Dict, Optional, Union, Any, List

from pydantic import BaseModel, validator
from pydantic.generics import GenericModel

from apps.crawler_platform.core_platform.core_g import CoreSqlValue

DataT = TypeVar('DataT')
DataX = TypeVar('DataX')


class InputPlatformModel(GenericModel, Generic[DataT]):
    input_code: str = ''
    input_msg: str = ''
    data: DataT


class journalHomeMode(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    home_rawid: str = ''
    page_index: int = 0


class journalInputMode(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    journal_rawid: str = ''


class journalVolumeMode(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    journal_rawid: str = ''
    pub_year: str = ''
    num: str = ''


class journalIssueMode(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    journal_rawid: str = ''
    pub_year: str = ''
    num: str = ''
    page_index: int = -1


class journalArticleMode(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    rawid: str = ''


class JournalArticleMode(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    rawid: list = []
    op: str = "merge"


class JournalArticleETLMode(JournalArticleMode):
    table_latest: dict = {}
    lngid: str = ''
    sub_db_id: str = ''
    data: dict = {}
    op: str = 'merge'  # 数据提交到远程etl接口合并方式,merge/overwrite


class JournalArticleETLValidateMode(journalArticleMode):
    lngid: str = ''
    sub_db_id: str = ''
    data: dict = {}
    # 多条数据，每条数据使用字典
    list_data: list = []


class JournalIssueInputMode(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    journal_rawid: str = ''


class HtmlModel(BaseModel):
    html: Optional[str] = None
    # 20250107 增加
    author_organ: Optional[str] = None
    state_dict: Any = ""
    down_date: str = str(datetime.now())


class PageHtmlModel(GenericModel, Generic[DataT]):
    """
    不要给默认值,否者使用Union会出现匹配不到model的情况
    """
    page_html: Dict[str, DataT]


class TypeHtmlModel(GenericModel, Generic[DataX]):
    """
    不要给默认值,否者使用Union会出现匹配不到model的情况
    """
    type_code: Dict[str, DataX]


class MultiplePageHtmlModel(GenericModel, Generic[DataT, DataX]):
    """
    不要给默认值,否者使用Union会出现匹配不到model的情况
    """
    first_page: DataT
    type_page: TypeHtmlModel[DataX]


T1 = Union[PageHtmlModel[HtmlModel], HtmlModel]
T2 = Union[MultiplePageHtmlModel[HtmlModel, T1], HtmlModel]


# T3 = Union[TypeHtmlModel[PageHtmlModel[HtmlModel]], HtmlModel]


# 下载的model
class DownModel(GenericModel, Generic[DataT]):
    down_dict: Dict[str, DataT]

    def set_parse_obj(self, obj):
        """
        解决载入数据是类方法的问题
        :param obj:
        :return:
        """
        return self.__class__.parse_obj(obj)


class ParaModel(BaseModel):
    # 解析规则字典
    parse_dict: dict = {}
    # 下载的html字典
    html_model: HtmlModel
    call_back: str = ""


class ParaAllModel(BaseModel):
    parse_dict: Dict[str, ParaModel] = {}


class DealInsertModel(BaseModel):
    lists: List[dict] = []
    insert_pre: str = ""


class DescriptionEnum(str, Enum):
    and_d = 'and'
    or_d = 'or'


class OperatorSqlModel(BaseModel):
    """
    sql的项由以下三部分组成
    """
    key: str = ""
    value: str = ""
    operator: str = ""
    description: DescriptionEnum = DescriptionEnum.and_d


class DealUpdateModel(BaseModel):
    # 兼容sql语句 +1 这样的操作（"update_no_placeholder":{"a":"a+1"}）
    update_no_placeholder: dict = {}
    # 正常更新的字典
    update: dict = {}
    # 这里做一个兼容，允许简单的字典，表示默认的等于和and组合 也可以用后面的List[OperatorSqlModel]构建复杂的sql
    # 这里原来使用dict，不想改太多东西 做的兼容，后面的代码全部使用 List[OperatorSqlModel]
    where: Union[dict, List[OperatorSqlModel]] = {}


class DealItemModel(BaseModel):
    # 写入上一级数据库的model
    # 兼容sql语句 +1 这样的操作（"update_no_placeholder":{"a":"a+1"}）
    update_no_placeholder: dict = {}
    # 正常更新的字典
    update: dict = {}
    # 条件更新能力
    update_list: List[DealUpdateModel] = []
    #  插入的语句,可能按照多个模式插入 比如insert 和 repalce
    insert: List[DealInsertModel] = []


class DealItemNextModel(BaseModel):
    update_list: List[DealUpdateModel] = []
    #  插入的语句,可能按照多个模式插入 比如insert 和 repalce
    insert: List[DealInsertModel] = []


class DealModel(BaseModel):
    # 写入上一级数据库的model
    befor_dicts: DealItemModel = DealItemModel()
    # 写入下一级数据库的字典, 可在callback覆盖
    next_dicts: DealItemNextModel = DealItemNextModel()
    # 多余的字典，可能是由于本次数据没有收录
    other_dicts: dict = {}
    # 没有数据的字典，网页上本条数据没有这个字段
    null_dicts: dict = {}
    # 特殊的字段，程序内部使用，不会写入数据库
    code_dicts: dict = {
        # 翻页时会启用该字段，告诉我解析出来的最大页
        "1_1": {"max_page": ""}  # 多个翻页时建议使用key区分
    }


class EtlItemModelQkA(BaseModel):
    pass


class EtlDealModel(BaseModel):
    status: str = ""
    err_msg: str = ""
    code = 9
    # save_data: dict = {}
    # extra_meta: dict = {}
    # ref_data: dict = {}
    ref_state: dict = {}
    save_data: List[dict] = []
    befor_dicts: DealItemModel = DealItemModel()
    other_info = {}  # 携带其他数据


# 数据库 taskinfo 对应的model，主要记录下载需要的信息
class TaskInfoModel(BaseModel):
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 这一步后下一步的tag，可以是多个;号分割
    task_tag_next: str = ""
    # 分组，加入分组是为了防止同一tag下的多路分支请求且相互不依赖的问题
    # 比如 期刊的海外版和国内版
    groups: str = ""
    # 同一组下的数据需要多个请求获取，且有可能需要先后顺序的依赖
    order_num: int = 1
    # url 请求的url地址
    url: str = ""
    # 请求的类型
    req_type: str = ""
    # 自己封装的一些中间件，进行验证
    resp_middler: str = ""
    # 请求头
    headers: str = ""
    # get的 请求参数
    params: str = ""
    # post的data参数
    post_data: str = ""
    # html中需要存在的字符串
    marks: str = ""
    # 解析规则
    parse_rules: str = ""
    # ETL解析规则
    etl_rules: Union[str, None] = ""
    # 验证规则
    validate_rules: str = ""
    # 请求超时
    timeout: int = ""
    # 自己写的方法 需要加到resp_middler中去的
    funcs: str = ""
    # 0: 代表没有翻页 1: 代表从0开始翻页 2: 代表从1开始翻页
    turn_page: int = 0
    # 任务打标键值对{"stat":"failcount"}
    state_key: str = ""
    # post 是不是json格式传输 1为是 0为不是
    post_data_is_json: str = ""
    # redis的更新时间
    redis_uptime: Optional[str] = ""
    # 说明
    introduction: str = ""
    # 其余参数
    others_parameter: str = "{}"
    # proxy 继续沿用标志, 2为单个任务一直用同一个代理直到失败
    proxy_state: int = 0


class TaskInfoSaveSettingChangeModel(BaseModel):
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 用作表示子任务所属表
    database_table_name: str = "taskinfo_save_setting"
    # taskinfo_save_setting表中的table_name字段
    table_name: str = ""
    # 用于确定本次请求归属, 0: 表示本次请求输入的参数 为关闭参数(即将输入tag_state或task_state置为0,其余置为1)
    # 1: 表示本次请求输入的参数 为开启参数(即将输入tag_state或task_state置为1, 其余置为0)
    switch: str = "1"
    # 用于控制本次请求是否为整个任务控制, 0: 表示本次请求不是以task_state 为整体(即将task_state置为1, 其余不做操作);
    # 1: 表示本次请求是控制整个任务是以task_state 为整体(即将task_state置为1, 其余置为0)
    is_task: str = "0"


# 保存nosql需要的配置
class TaskInfoSaveSettingModel(BaseModel):
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 需要哪些sql里的键组成 nosql的主键
    id_list: str = ""
    # nosql 的表名，在后台表明统一管理，不用担心跨库问题
    nosql_table: str = ""
    # 保存历史记录个数
    history_cnt: int = 1
    # redis的更新时间
    redis_uptime: Optional[str] = ""
    # 用作表示任务总体下载控制
    task_state: int = 0
    # 用作表示子任务总体下载控制
    tag_state: int = 0
    # 用作表示子任务所属表
    table_name: str = ""
    # 子任务所属表 etl查询索引名
    table_index: str = ""
    # 用作表示子任务对象字段
    final_stat: str = ""
    # 用作表示单个任务代理单位时间请求次数
    proxy_ctrl: int = 0
    # 用于表示所需代理的种类
    proxy_type: str = ""
    # 说明
    introduction: str = ''
    # 平台对应的api
    api_url: str = ''
    # 该任务对应发送api的model
    model_name: str = ''
    # sql 数据的model name
    sql_model_name: str = ''
    # 该任务需要执行的sql语句
    sql_exc: str = ''
    # sub_db_id
    sub_db_id: str = ''
    # 专为article级设定, 以便于后面取HTML
    table_latest: str = ''
    force_index: str = ''
    dispatch_model_name: str = ''

    def get_id_list(self):
        return eval(self.id_list)


class TaskAllModel(BaseModel):
    """
    将两个redis集成到一个model
    """
    task_set: TaskInfoSaveSettingModel
    task_info: TaskInfoModel


class RedisAllTaskModel(BaseModel):
    """
    由于task_set 和 task_info是一对多的关系,所以TaskAllModel中的task_set会重复
    字典键 str为1_1 即 数据库taskinfo中的groups_ordernum
    """
    parse_dict: Dict[str, TaskAllModel] = {}


# 期刊列表model
class JournalListModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 某个网站期刊的主键
    journal_rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    # json格式的str,期刊的其他信息
    journal_json: str = ""
    # 是否存活
    is_active: int = 1
    # 状态
    state: int = 0
    failcount: int = 0
    online_stat: int = 0
    online_failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""

    @validator('null_dicts', 'other_dicts', 'err_msg', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        journal_json = json.loads(dicts["journal_json"])
        dicts.update(journal_json)
        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        journal_json = json.loads(dicts["journal_json"])
        journal_json.update(up_dicts)
        self.journal_json = json.dumps(journal_json, ensure_ascii=False)


# 期刊 卷 model
class JournalVolumeModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 某个网站期刊的主键
    journal_rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    # 出版年
    pub_year: str = ""
    # 卷
    num: str = ""
    volume_json: str = ""
    # 页
    page: str = ""
    # 状态
    state: int = 0
    failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""

    @validator('null_dicts', 'other_dicts', 'err_msg', 'page', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        journal_json = json.loads(dicts["volume_json"])
        dicts.update(journal_json)
        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        journal_json = json.loads(dicts["volume_json"])
        journal_json.update(up_dicts)
        self.volume_json = json.dumps(journal_json, ensure_ascii=False)


# 期列表model
class JournalIssueModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 某个网站期刊的主键
    journal_rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    # 出版年
    pub_year: str = ""
    # 期
    num: str = ""
    # json格式的str,期的其他信息, 目前仅超星使用 ZCN
    issue_json: str = ""
    # 页
    page: str = ""
    # 当前页
    page_index: int = -1

    # 状态
    state: int = 0
    failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""

    @validator('null_dicts', 'other_dicts', 'err_msg', 'page', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        journal_json = json.loads(dicts["issue_json"])
        dicts.update(journal_json)
        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        journal_json = json.loads(dicts["issue_json"])
        journal_json.update(up_dicts)
        self.issue_json = json.dumps(journal_json, ensure_ascii=False)


class CallBackModel(GenericModel, Generic[DataT]):
    # 通过xpath或者css 、json解析出来的字典
    para_dicts: dict = {}
    # sql查询出来的Model
    sql_model: DataT
    # redis所有配置信息
    redis_all: RedisAllTaskModel
    # # html
    # down_model: DownModel # 改动比较大暂时不获取


class EtlCallBackModel(GenericModel, Generic[DataT, DataX]):
    # 通过xpath或者css 、json解析出来的字典
    para_dicts: dict = {}
    down_model: DownModel[DataX]
    # sql查询出来的Model
    sql_model: DataT
    # redis所有配置信息
    redis_all: RedisAllTaskModel


# 期列表model
class JournalArticleModel(BaseModel):
    # 自增主id
    id: int
    task_name: str = ""
    task_tag: str = ""
    # 某个网站期刊的主键
    rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    article_info_json: str = ""
    article_json: str = ""
    state: int = ""
    failcount: int = ""
    ref_stat: int = ""
    ref_failcount: int = 0
    # 状态
    oversea: int = 0
    oversea_failcount: int = 0
    re_down: int = ""
    is_true: int = ""
    is_true_failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""
    page: str = "0"

    @validator('null_dicts', 'other_dicts', 'err_msg', pre=True)
    def split_str(cls, v):
        """
        pre=True 是否应该在标准验证器之前调用该验证器(否则在标准验证器之后调用)
        :param v:
        :return:
        """
        if v is None:
            return ""
        return v

    def set_json(self, up_dicts):
        dicts = self.dict()
        article_info_json = dicts.get("article_info_json", "")
        if len(article_info_json) == 0:
            article_info_json = dicts.get("article_json", "")
        journal_json = json.loads(article_info_json)
        journal_json.update(up_dicts)
        self.article_info_json = json.dumps(journal_json, ensure_ascii=False)

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        article_info_json = dicts.get("article_info_json", "")
        if len(article_info_json) == 0:
            article_info_json = dicts.get("article_json", "")
        journal_json = json.loads(article_info_json)
        dicts.update(journal_json)
        return dicts


# ETL 期刊文章
class JournalEtlArticleModel(BaseModel):
    task_name: str = ""
    task_tag: str = ""
    # 某个网站期刊的主键
    rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    article_info_json: str = ""
    article_json: str = ""
    state: int = ""
    failcount: int = ""
    ref_stat: int = ""
    ref_failcount: int = 0
    # 状态
    oversea: int = 0
    oversea_failcount: int = 0
    re_down: int = ""
    is_true: int = ""
    is_true_failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""
    page: str = "0"

    @validator('null_dicts', 'other_dicts', 'err_msg', pre=True)
    def split_str(cls, v):
        """
        pre=True 是否应该在标准验证器之前调用该验证器(否则在标准验证器之后调用)
        :param v:
        :return:
        """
        if v is None:
            return ""
        return v

    def set_json(self, up_dicts):
        dicts = self.dict()
        article_info_json = dicts.get("article_info_json", "")
        if len(article_info_json) == 0:
            article_info_json = dicts.get("article_json", "")
        journal_json = json.loads(article_info_json)
        journal_json.update(up_dicts)
        self.article_info_json = json.dumps(journal_json, ensure_ascii=False)

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        article_info_json = dicts.get("article_info_json", "")
        if len(article_info_json) == 0:
            article_info_json = dicts.get("article_json", "")
        journal_json = json.loads(article_info_json)
        dicts.update(journal_json)
        return dicts
    journal_info: dict = {}


# 期刊首页model
class JournalHomeModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 某个网站的主键
    home_rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    page_index: str = ""

    # json格式的str,
    home_json: str = ""
    # 页
    page: str = ""
    # 状态
    state: int = 0
    failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""

    @validator('null_dicts', 'other_dicts', 'err_msg', 'page', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        home_json = json.loads(dicts["home_json"])
        if home_json is None:
            home_json = {}
        dicts.update(home_json)

        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        journal_json = json.loads(dicts["home_json"])
        journal_json.update(up_dicts)
        self.home_json = json.dumps(journal_json, ensure_ascii=False)


# ############################## 以下  paper

class PaperDaysMode(BaseModel):
    task_name: str = ""
    task_tag: str = ""
    year: str = ""
    months: str = ""
    days: str = ""


class PaperDaysModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    task_tag: str = ""
    year: str = ""
    months: str = ""
    days: str = ""
    url: str = ""
    # json格式的str,
    days_json: str = ""
    # 状态
    state: int = 0
    failcount: int = 0
    count: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        days_json = json.loads(dicts["days_json"])
        dicts.update(days_json)
        return dicts


class PaperPageMode(BaseModel):
    task_name: str = ""
    task_tag: str = ""
    rawid_page: str = ""


class PaperPageModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    task_tag: str = ""
    rawid_page: str = ""

    # 状态
    state: int = 0
    failcount: int = 0
    count: int = 0
    create_time: datetime
    update_time: datetime
    # json格式的str,
    page_json: str = ""
    other_dicts: Optional[str] = None
    null_dicts: Optional[str] = None
    err_msg: Optional[str] = None

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        days_json = json.loads(dicts["page_json"])
        dicts.update(days_json)
        return dicts


class PaperArticleMode(BaseModel):
    task_name: str = ""
    task_tag: str = ""
    rawid: str = ""


class PaperArticleModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    task_tag: str = ""
    rawid: str = ""

    # 状态
    state: int = 0
    failcount: int = 0
    count: int = 0
    create_time: datetime
    update_time: datetime
    # json格式的str,
    article_json: str = ""
    other_dicts: Optional[str] = None
    null_dicts: Optional[str] = None
    err_msg: Optional[str] = None

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        days_json = json.loads(dicts["article_json"])
        dicts.update(days_json)
        return dicts


class SaveSqlite3(BaseModel):
    task_tag: str = ''
    data_list: List[dict] = []
    int_list: List = []


class SaveMdb(BaseModel):
    task_tag: str = ''
    data_list: List[dict] = []
    int_list: List = []
    long_text_list: List = []


class CnkiMarkTable(BaseModel):
    level: str = ''
    journal_rawid: str = ''
    years_num_list: List = []
    rawid_list: List = []


class QkUpdateManger(BaseModel):
    journal_id: str = ''
    table_name: str = ''
    sql_exc: str = ''


class SaveManageTaskSettingModel(BaseModel):
    journal_id: str = ''
    table_name: str = ''
    sql_exc: str = ''
    delay_type: str = ''
    delay_time: int = 1
    counts: int = 0
    # types open指打开该任务, close指关闭该任务
    types: str = 'open'
    # state 用于表示该任务当前是否需要更新, 0为需要, 1为更新完成, 默认为0
    state: int = 0


# 百科 词条
class BaikeHomeModel(BaseModel):
    # 自增主id
    id: int
    task_name: str = ""
    task_tag: str = ""
    # 某个网站的主键
    home_rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    # json格式的str,
    home_json: str = ""
    # 状态
    state: int = 0
    failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""

    @validator('null_dicts', 'other_dicts', 'err_msg', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        home_json = json.loads(dicts["home_json"])
        if home_json is None:
            home_json = {}
        dicts.update(home_json)

        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        journal_json = json.loads(dicts["home_json"])
        journal_json.update(up_dicts)
        self.home_json = json.dumps(journal_json, ensure_ascii=False)


class BaikeArticleModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    task_tag: str = ""
    rawid: str = ""

    # 状态
    state: int = 0
    failcount: int = 0
    count: int = 0
    create_time: datetime
    update_time: datetime
    # json格式的str,
    article_json: str = ""
    other_dicts: Optional[str] = None
    null_dicts: Optional[str] = None
    err_msg: Optional[str] = None

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        days_json = json.loads(dicts["article_json"])
        dicts.update(days_json)
        return dicts


class BaikeHomeMode(BaseModel):
    task_name: str = ""
    task_tag: str = ""
    home_rawid: str = ""


class BaikeArticleMode(BaseModel):
    task_name: str = ""
    task_tag: str = ""
    rawid: str = ""


class FileGetFromWeb(str, Enum):
    chaoxing_list = "chaoxingjournal_chaoxingjournallist"
    cnki_list = "cnkijournal_cnkijournallist"
    wanfang_list = "wanfangjournal_wanfangjournallist"
    cnki_subject_list = "cnki_subject_list"
    cnki_subject_map = "cnki_subject_map"


#########  政策
class PolicyListModel(BaseModel):
    # 自增主id
    id: int
    task_name: str = ""
    task_tag: str = ""
    list_rawid: str = ""
    sub_db_id: str = ""
    list_json: str = ""
    page_index: int = 0
    # 状态
    state: int = 0
    failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""

    @validator('null_dicts', 'other_dicts', 'err_msg', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        home_json = json.loads(dicts["list_json"])
        if home_json is None:
            home_json = {}
        dicts.update(home_json)

        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        journal_json = json.loads(dicts["list_json"])
        journal_json.update(up_dicts)
        self.home_json = json.dumps(journal_json, ensure_ascii=False)


class PolicyArticleModel(BaseModel):
    # 自增主id
    id: int
    task_name: str = ""
    task_tag: str = ""
    rawid: str = ""
    state: int = 0
    failcount: int = 0
    count: int = 0
    create_time: datetime
    update_time: datetime
    article_json: str = ""
    other_dicts: Optional[str] = None
    null_dicts: Optional[str] = None
    err_msg: Optional[str] = None

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        days_json = json.loads(dicts["article_json"])
        dicts.update(days_json)
        return dicts


class PolicyListMode(BaseModel):
    task_name: str = ""
    task_tag: str = ""
    list_rawid: str = ""
    page_index: str = ""


class PolicyArticleMode(BaseModel):
    task_name: str = ""
    task_tag: str = ""
    rawid: str = ""


class LobInsert(BaseModel):
    table_name: str = ""
    rawid: str = ""
    datas: str = ""
    # 在插入数据存在的情况下是否允许进行replace
    is_replace: str = "True"


class LobDelete(BaseModel):
    table_name: str = ""
    rawid: str = ""


class LobReplace(BaseModel):
    table_name: str = ""
    rawid: str = ""
    datas: str = ""


class OtherHomeModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 某个网站的主键
    list_rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    page_index: int = ""

    # json格式的str,
    home_json: str = ""
    # 页
    page: int = ""
    # 状态
    state: int = 0
    failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""
    is_active: int = 0

    @validator('null_dicts', 'other_dicts', 'err_msg', 'page', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        home_json = json.loads(dicts["home_json"])
        if home_json is None:
            home_json = {}
        dicts.update(home_json)

        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        home_json = json.loads(dicts["home_json"])
        home_json.update(up_dicts)
        self.home_json = json.dumps(home_json, ensure_ascii=False)


# 期刊列表model
class OtherListModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 某个网站期刊的主键
    list_rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    # json格式的str,期刊的其他信息
    list_json: str = ""
    # 是否存活
    is_active: int = 1
    # 状态
    state: int = 0
    failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""
    page: int = 0
    page_index: int = 0

    @validator('other_dicts', 'null_dicts', 'err_msg', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        list_json = json.loads(dicts["list_json"])
        dicts.update(list_json)
        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        list_json = json.loads(dicts["list_json"])
        list_json.update(up_dicts)
        self.list_json = json.dumps(list_json, ensure_ascii=False)


class OtherInputMode(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    list_rawid: str = ''
    page_index: int = 1


# 期刊列表model
class OtherArticleModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 某个网站期刊的主键
    rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    # json格式的str,期刊的其他信息
    article_json: str = ""
    # 状态
    state: int = 0
    count: str = ""
    failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""
    page: str = ""

    @validator('other_dicts', 'null_dicts', 'err_msg', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        article_json = json.loads(dicts["article_json"])
        dicts.update(article_json)
        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        article_json = json.loads(dicts["article_json"])
        article_json.update(up_dicts)
        self.article_json = json.dumps(article_json, ensure_ascii=False)


class OtherArticleEtlModel(OtherArticleModel):
    journal_info: dict = {}


class OtherArticleMode(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    rawid: str = ''


class OtherArticleSpeModel(BaseModel):
    # 自增主id
    id: int
    task_name: str = ""
    task_tag: str = ""
    # 某个网站期刊的主键
    rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    article_info_json: str = ""
    article_json: str = ""
    state: int = ""
    failcount: int = ""
    ref_stat: int = ""
    ref_failcount: int = 0
    is_true: int = ""
    is_true_failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""
    page: str = "0"

    @validator('null_dicts', 'other_dicts', 'err_msg', pre=True)
    def split_str(cls, v):
        """
        pre=True 是否应该在标准验证器之前调用该验证器(否则在标准验证器之后调用)
        :param v:
        :return:
        """
        if v is None:
            return ""
        return v

    def set_json(self, up_dicts):
        dicts = self.dict()
        article_info_json = dicts.get("article_info_json", "")
        if len(article_info_json) == 0:
            article_info_json = dicts.get("article_json", "")
        journal_json = json.loads(article_info_json)
        journal_json.update(up_dicts)
        self.article_info_json = json.dumps(journal_json, ensure_ascii=False)

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        article_info_json = dicts.get("article_info_json", "")
        if len(article_info_json) == 0:
            article_info_json = dicts.get("article_json", "")
        journal_json = json.loads(article_info_json)
        dicts.update(journal_json)
        return dicts


class OtherArticleSpeEtlModel(OtherArticleSpeModel):
    journal_info: dict = {}


class GetArticleHtml(BaseModel):
    lngid: str = ''


class GetArticleHtmlLinShi(BaseModel):
    lngid_list: List = []
    sub_db_id: str = ''
    get_dict: dict = {}


class GetJournalHtml(BaseModel):
    id_list: List = []
    journal_info: dict = {}


class GetCommanderHtml(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    sub_db_id: str = ''
    # 告诉需要提取的html的类型 list issue article
    get_type: str = 'not_latest'
    infos: dict = {}
    id_list: List[dict] = []


class OtherSubclassModel(BaseModel):
    # 自增主id
    id: int
    # 任务名，一般一个网站一个任务名
    task_name: str = ""
    # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
    # 当然也兼容一些特殊的情况，比如海外版,网络首发等
    task_tag: str = ""
    # 某个网站期刊的主键
    list_rawid: str = ""
    # 维普定义的某个库的id
    sub_db_id: str = ""
    # json格式的str,期刊的其他信息
    subclass_json: str = ""
    # 是否存活
    is_active: int = 1
    # 状态
    state: int = 0
    failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""
    page: int = 0
    page_index: int = 0

    @validator('other_dicts', 'null_dicts', 'err_msg', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        subclass_json = json.loads(dicts["subclass_json"])
        dicts.update(subclass_json)
        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        subclass_json = json.loads(dicts["subclass_json"])
        subclass_json.update(up_dicts)
        self.subclass_json = json.dumps(subclass_json, ensure_ascii=False)


class NssdListMode(BaseModel):
    task_name: str = ""
    task_tag: str = ""
    list_rawid: str = ""
    page_index: str = ""


class NssdListModel(BaseModel):
    # 自增主id
    id: int
    task_name: str = ""
    task_tag: str = ""
    list_rawid: str = ""
    sub_db_id: str = ""
    list_json: str = ""
    page_index: str = ""
    # 状态
    state: int = 0
    failcount: int = 0
    create_time: datetime
    update_time: datetime
    other_dicts: str = ""
    null_dicts: str = ""
    err_msg: str = ""

    @validator('null_dicts', 'other_dicts', 'err_msg', pre=True)
    def split_str(cls, v):
        if v is None:
            return ""
        return v

    def get_dict(self):
        """
        将json数据扩展到外层使用
        :return:
        """
        dicts = self.dict()
        home_json = json.loads(dicts["list_json"])
        if home_json is None:
            home_json = {}
        dicts.update(home_json)

        return dicts

    def set_json(self, up_dicts):
        dicts = self.dict()
        journal_json = json.loads(dicts["list_json"])
        journal_json.update(up_dicts)
        self.home_json = json.dumps(journal_json, ensure_ascii=False)


class EtlParseMode(BaseModel):
    task_name: str = ''
    task_tag: str = ''
    id_dicts: dict = {}


class ModelColl(object):

    @classmethod
    def get_html_model(cls, name):
        """
        获取html对应的model
        :param name:
        :return:
        """
        if name.strip() == "" or name is None:
            raise Exception("model name 不能为空")
        return eval(name)

    @classmethod
    def get_sql_model(cls, name):
        """
        获取sql 对应的model
        :param name:
        :return:
        """
        return eval(name)


class ChaoXingIssueHtml(BaseModel):
    id_list: List = []


class FlowAirActive(BaseModel):
    # airflow里面的dag_id
    dag_id: str = ''
    state: str = ''
    is_active: str = ''
    # 标识这个任务应该在哪台服务器上启动
    main_server: str = ''
    # 表示airflow任务当前是否暂停, 默认为 没暂停
    is_paused: int = 0
    need_active: str = 'open'
    task_author: str = 'caiji'
    task_manager: str = 'caiji'
    # 如果需要对失败任务进行发任务通知,设为1, 针对一些间隔周期长的任务
    # 间隔周期长, 则感知不明显, 该功能给予感知
    failed_notice: int = 0
    # 增加介绍, 比如该任务因为什么暂停或者各种原因, 给有需要的人准备, 默认为空
    introduce: str = ''
    relation: str = ''


class PolicyUrlConn(BaseModel):
    url: str = ''

class IsticTasks(BaseModel):
    now_date: str = ''
    batch: str = ''

# # Medjournals 中华医学期刊网
# class MedjournalsYearsModel(BaseModel):
#     # 自增主id
#     id: int
#     task_name: str = ""
#     task_tag: str = ""
#     years_rawid: str = ""
#     sub_db_id: str = ""
#     years_json: str = ""
#     years: str = ""
#     # 状态
#     state: int = 0
#     failcount: int = 0
#     create_time: datetime
#     update_time: datetime
#     other_dicts: str = ""
#     null_dicts: str = ""
#     err_msg: str = ""
#
#     @validator('null_dicts', 'other_dicts', 'err_msg', pre=True)
#     def split_str(cls, v):
#         if v is None:
#             return ""
#         return v
#
#     def get_dict(self):
#         """
#         将json数据扩展到外层使用
#         :return:
#         """
#         dicts = self.dict()
#         years_json = json.loads(dicts["years_json"])
#         if years_json is None:
#             years_json = {}
#         dicts.update(years_json)
#
#         return dicts
#
#     def set_json(self, up_dicts):
#         dicts = self.dict()
#         journal_json = json.loads(dicts["years_json"])
#         journal_json.update(up_dicts)
#         self.years_json = json.dumps(journal_json, ensure_ascii=False)
#
#
# class MedjournalsYearsMode(BaseModel):
#     task_name: str = ""
#     task_tag: str = ""
#     years_rawid: str = ""
#     years: str = ""
#
#
# class MedjournalsIssueModel(BaseModel):
#     # 自增主id
#     id: int
#     # 任务名，一般一个网站一个任务名
#     task_name: str = ""
#     # 任务标签，一般一个步骤一个任务标签，比如期刊分为cnkijournallist，cnkiissue，cnkiarticle
#     # 当然也兼容一些特殊的情况，比如海外版,网络首发等
#     task_tag: str = ""
#     # 某个网站期刊的主键
#     journal_rawid: str = ""
#     # 维普定义的某个库的id
#     sub_db_id: str = ""
#     # 出版年
#     pub_year: str = ""
#     # 期
#     num: str = ""
#     # json格式的str,期的其他信息, 目前仅超星使用 ZCN
#     issue_json: str = ""
#     # 页
#     # 状态
#     state: int = 0
#     failcount: int = 0
#     create_time: datetime
#     update_time: datetime
#     other_dicts: str = ""
#     null_dicts: str = ""
#     err_msg: str = ""
#
#     def get_dict(self):
#         """
#         将json数据扩展到外层使用
#         :return:
#         """
#         dicts = self.dict()
#         journal_json = json.loads(dicts["issue_json"])
#         dicts.update(journal_json)
#         return dicts
#
#     def set_json(self, up_dicts):
#         dicts = self.dict()
#         journal_json = json.loads(dicts["issue_json"])
#         journal_json.update(up_dicts)
#         self.issue_json = json.dumps(journal_json, ensure_ascii=False)
#
#
# class MedjournalsIssueMode(BaseModel):
#     task_name: str = ""
#     task_tag: str = ""
#     journal_rawid: str = ""
#     pub_year: str = ""
#     num: str = ""
#
#
# class MedjournalsArticleModel(BaseModel):
#     # 自增主id
#     id: int
#     task_name: str = ""
#     task_tag: str = ""
#     rawid: str = ""
#     state: int = 0
#     failcount: int = 0
#     count: int = 0
#     create_time: datetime
#     update_time: datetime
#     article_json: str = ""
#     other_dicts: Optional[str] = None
#     null_dicts: Optional[str] = None
#     err_msg: Optional[str] = None
#
#     def get_dict(self):
#         """
#         将json数据扩展到外层使用
#         :return:
#         """
#         dicts = self.dict()
#         days_json = json.loads(dicts["article_json"])
#         dicts.update(days_json)
#         return dicts
#
#
# class MedjournalsArticleMode(BaseModel):
#     task_name: str = ""
#     task_tag: str = ""
#     rawid: str = ""
class IsticData(BaseModel):
    datas: Dict
    dele_batch: str = ""

class CnkiPdf(BaseModel):
    journal_rawid: str = ""
    pub_year: str = ""
    num: str = ""